package com.ww.crawler.webmagic;

import java.util.List;

import com.ww.crawler.webmagic.pipeline.ImgPipeline;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class JointourPageProcessor implements PageProcessor {
	// 抓取网站的相关配置，包括：编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(10).setSleepTime(1000)
		    .addHeader("Connection", "keep-alive")
		    .addHeader("Referer", "https://www.jointour.cn/")
		    .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0")
		    .setCharset("UTF-8").enableHttpProxyPool();
	
	@Override
	public void process(Page page) {
//		List<String> listProcess = new ArrayList<>();
		page.addTargetRequests(page.getHtml().links()
				.regex("(.*(jointour)+.*)")
				.all());
		
		List<String> listProcess = page.getHtml().links().regex("(.*(jpg|png)$)").all();
		if(listProcess.size() > 0){
			page.putField("img", listProcess);
		}
//		if(page.getUrl().regex("((jpg|png)$)").match()){
//			num++;
//			listProcess.add(page.getHtml().get());
//			System.out.println(">>>>>>>>>>>>> " + num);
//		}
	}

	@Override
	public Site getSite() {
		return this.site;
	}
	
	public static void main(String[] args) {
		
		Spider.create(new JointourPageProcessor()).addUrl("http://www.jointour.cn/home/home.action").addPipeline(new ImgPipeline())
		// FileCacheQueueScheduler 使用文件缓存作增量爬取
//		.setScheduler(new FileCacheQueueScheduler("/Users/wangwei/soft/webmagic_jointour"))
		.thread(5).run();
	}
}
